This notebook contains the key visualizations for the Task Mapping
paper.
library(factoextra)
library(NbClust)
library(cluster)
library(plotly)
library(ggplot2)
library(caret) #for knn
library(e1071) #for svm
library(dplyr)
library(tidyverse)
Very useful decision boundary plotting code from: https://mhahsler.github.io/Introduction_to_Data_Mining_R_Examples/book/classification-alternative-techniques.html#k-nearest-neighbors
decisionplot <- function(model, data, class_var,
predict_type = c("class", "prob"), resolution = 5 * 75) {
# resolution is set to 75 dpi if the image is rendered 5 inces wide.
y <- data %>% pull(class_var)
x <- data %>% dplyr::select(-all_of(class_var))
# resubstitution accuracy
prediction <- predict(model, x, type = predict_type[1])
# LDA returns a list
if(is.list(prediction)) prediction <- prediction$class
prediction <- factor(prediction, levels = levels(y))
cm <- confusionMatrix(data = prediction, reference = y)
acc <- cm$overall["Accuracy"]
# evaluate model on a grid
r <- sapply(x[, 1:2], range, na.rm = TRUE)
xs <- seq(r[1,1], r[2,1], length.out = resolution)
ys <- seq(r[1,2], r[2,2], length.out = resolution)
g <- cbind(rep(xs, each = resolution), rep(ys, time = resolution))
colnames(g) <- colnames(r)
g <- as_tibble(g)
### guess how to get class labels from predict
### (unfortunately not very consistent between models)
cl <- predict(model, g, type = predict_type[1])
# LDA returns a list
if(is.list(cl)) {
prob <- cl$posterior
cl <- cl$class
} else
try(prob <- predict(model, g, type = predict_type[2]))
# we visualize the difference in probability/score between the
# winning class and the second best class.
# don't use probability if predict for the classifier does not support it.
max_prob <- 1
try({
max_prob <- t(apply(prob, MARGIN = 1, sort, decreasing = TRUE))
max_prob <- max_prob[,1] - max_prob[,2]
}, silent = TRUE)
cl <- factor(cl, levels = levels(y))
g <- g %>% add_column(prediction = cl, probability = max_prob)
ggplot(g, mapping = aes_string(
x = colnames(g)[1],
y = colnames(g)[2])) +
geom_raster(mapping = aes(fill = prediction, alpha = probability)) +
geom_contour(mapping = aes(z = as.numeric(prediction)),
bins = length(levels(cl)), size = .5, color = "black") +
geom_point(data = data, mapping = aes_string(
x = colnames(data)[1],
y = colnames(data)[2],
shape = class_var), alpha = .7) +
scale_alpha_continuous(range = c(0,1), limits = c(0,1), guide = "none") +
labs(subtitle = paste("Training accuracy:", round(acc, 2)))
}
Load the Data
task_map <- read_csv('../task_map.csv')
Rows: 102 Columns: 24-- Column specification ----------------------------------------------------------------------------------------
Delimiter: ","
chr (1): task
dbl (23): Q1concept_behav, Q3type_1_planning, Q4type_2_generate, Q6type_5_cc, Q7type_7_battle, Q8type_8_perf...
i Use `spec()` to retrieve the full column specification for this data.
i Specify the column types or set `show_col_types = FALSE` to quiet this message.
Plot the Task Map and other Related Images
Draw the task map using PCA & clustering
First, run the PCA
set.seed(1)
pca <- task_map %>% #select(-continuous_questions) %>%
select(-task) %>%
prcomp(center = T)
# get optimal number of clusters -- "silhouette" method
fviz_nbclust(x = pca$x, FUNcluster = stats::kmeans, method = "silhouette") +
labs(subtitle = "Silhouette method")

# get optimal number of clusters
NbClust(data = pca$x, distance = "euclidean",
min.nc = 2, max.nc = 15, method = "kmeans")
*** : The Hubert index is a graphical method of determining the number of clusters.
In the plot of Hubert index, we seek a significant knee that corresponds to a
significant increase of the value of the measure i.e the significant peak in Hubert
index second differences plot.

*** : The D index is a graphical method of determining the number of clusters.
In the plot of D index, we seek a significant knee (the significant peak in Dindex
second differences plot) that corresponds to a significant increase of the value of
the measure.
*******************************************************************
* Among all indices:
* 8 proposed 2 as the best number of clusters
* 4 proposed 3 as the best number of clusters
* 2 proposed 4 as the best number of clusters
* 1 proposed 5 as the best number of clusters
* 2 proposed 7 as the best number of clusters
* 1 proposed 8 as the best number of clusters
* 1 proposed 9 as the best number of clusters
* 1 proposed 10 as the best number of clusters
* 1 proposed 11 as the best number of clusters
* 2 proposed 14 as the best number of clusters
* 1 proposed 15 as the best number of clusters
***** Conclusion *****
* According to the majority rule, the best number of clusters is 2
*******************************************************************
$All.index
KL CH Hartigan CCC Scott Marriot TrCovW TraceW Friedman Rubin Cindex DB
2 5.0069 61.8549 15.6938 0.5233 220.2327 24147.8630 41.3638 86.4849 6.2479 1.6185 0.4571 1.2204
3 1.1136 43.1918 14.4091 0.1053 376.1613 11780.1299 28.8616 74.7532 8.5042 1.8726 0.4356 1.6546
4 1.8754 37.4068 8.5834 0.9450 564.3520 3309.4161 20.8908 65.2555 15.0669 2.1451 0.4371 1.5369
5 0.9756 32.3249 8.7802 1.0947 661.2027 2000.7902 17.3124 60.0004 17.5862 2.3330 0.4476 1.4305
6 1.1393 29.6479 7.9923 1.6847 764.8096 1043.3448 13.2934 55.0201 18.7310 2.5442 0.4157 1.4520
7 1.6179 27.8029 -2.8124 2.0547 879.0917 463.1614 10.7668 50.7915 20.6066 2.7560 0.3940 1.4904
8 0.2936 22.4846 16.2947 -0.9150 834.8846 933.1268 11.7208 52.3410 19.8582 2.6744 0.4245 1.6856
9 2.6595 24.8540 6.9196 2.2594 1059.3556 130.7662 7.7355 44.6083 26.6681 3.1380 0.3680 1.5430
10 2.3249 24.2416 3.4172 2.7838 1180.4019 49.2743 6.3340 41.5191 31.8821 3.3715 0.3517 1.5127
11 0.9576 22.7199 3.5122 2.1276 1304.8645 17.5983 6.3698 40.0322 36.8754 3.4967 0.3293 1.4787
12 0.6126 21.5316 5.4211 1.6851 1372.6428 10.7761 5.5888 38.5445 34.0191 3.6316 0.3909 1.5414
13 1.1321 21.1404 4.9644 1.8342 1453.7129 5.7122 5.1333 36.3547 36.6380 3.8504 0.3623 1.3717
14 1.5224 20.7488 3.4737 1.9774 1614.4163 1.3707 4.4859 34.4340 45.9769 4.0652 0.3659 1.3532
15 0.8365 20.0450 4.1119 1.5082 1695.3182 0.7119 4.2980 33.1264 46.8036 4.2256 0.3627 1.3665
Silhouette Duda Pseudot2 Beale Ratkowsky Ball Ptbiserial Frey McClain Dunn Hubert SDindex Dindex
2 0.3416 0.7859 17.4368 4.3179 0.0630 43.2424 0.6395 1.5407 0.5524 0.2576 0.0177 2.6413 0.8811
3 0.2475 0.9032 4.1780 1.6751 0.0989 24.9177 0.5615 0.0573 1.2348 0.2463 0.0180 3.1665 0.8164
4 0.2392 3.4388 -24.1128 -9.9884 0.1191 16.3139 0.6027 0.5508 1.3885 0.3242 0.0229 3.0450 0.7672
5 0.2363 1.0237 -0.6933 -0.3582 0.1232 12.0001 0.5974 0.5295 1.5411 0.3370 0.0229 2.8657 0.7359
6 0.2284 0.7922 9.1811 4.1016 0.1316 9.1700 0.5751 1.1979 2.0372 0.2865 0.0241 3.0128 0.7079
7 0.1996 1.6517 -7.4970 -5.4438 0.1359 7.2559 0.5114 -0.7349 2.8564 0.2395 0.0255 3.5435 0.6770
8 0.1472 3.6616 -11.6303 -9.3601 0.1314 6.5426 0.4167 -0.0694 3.9377 0.2289 0.0227 3.9793 0.6884
9 0.2057 2.6105 -12.9556 -8.2752 0.1397 4.9565 0.4635 0.2250 3.8563 0.2228 0.0262 3.7203 0.6326
10 0.1991 2.9698 -13.9289 -8.5409 0.1412 4.1519 0.4546 0.2361 4.3147 0.2268 0.0272 3.6939 0.6094
11 0.1963 1.4043 -2.8789 -3.8615 0.1419 3.6393 0.4470 -1.6112 4.6482 0.1818 0.0277 3.6737 0.5981
12 0.1789 1.5077 -5.7246 -4.9686 0.1432 3.2120 0.4095 -0.2396 5.5124 0.2067 0.0281 3.8811 0.5862
13 0.2014 0.9828 0.2792 0.2633 0.1435 2.7965 0.4372 1.0361 5.0282 0.2030 0.0282 3.4167 0.5716
14 0.1982 0.7026 2.1168 5.4516 0.1457 2.4596 0.4140 0.2069 5.7059 0.2758 0.0287 3.7517 0.5534
15 0.2053 3.0324 -6.0321 -7.1920 0.1450 2.2084 0.4072 -0.0466 6.0546 0.2818 0.0291 3.5052 0.5439
SDbw
2 0.4333
3 0.3542
4 0.3121
5 0.2577
6 0.2645
7 0.2390
8 0.2303
9 0.2057
10 0.1909
11 0.1943
12 0.1785
13 0.1733
14 0.1523
15 0.1491
$All.CriticalValues
CritValue_Duda CritValue_PseudoT2 Fvalue_Beale
2 0.8582 10.5718 0.0000
3 0.8178 8.6863 0.0248
4 0.6634 17.2534 1.0000
5 0.7972 7.6335 1.0000
6 0.8178 7.7954 0.0000
7 0.6446 10.4743 1.0000
8 0.5939 10.9390 1.0000
9 0.6220 12.7618 1.0000
10 0.5939 14.3574 1.0000
11 0.6220 6.0771 1.0000
12 0.7153 6.7676 1.0000
13 0.7475 5.4039 0.9998
14 0.5939 3.4184 0.0000
15 0.5089 8.6854 1.0000
$Best.nc
KL CH Hartigan CCC Scott Marriot TrCovW TraceW Friedman Rubin Cindex DB
Number_clusters 2.0000 2.0000 8.0000 10.0000 9.000 4.000 3.0000 7.0000 14.0000 7.0000 11.0000 2.0000
Value_Index 5.0069 61.8549 19.1071 2.7838 224.471 7162.088 12.5022 5.7781 9.3389 -0.2934 0.3293 1.2204
Silhouette Duda PseudoT2 Beale Ratkowsky Ball PtBiserial Frey McClain Dunn Hubert
Number_clusters 2.0000 3.0000 3.000 4.0000 14.0000 3.0000 2.0000 2.0000 2.0000 5.000 0
Value_Index 0.3416 0.9032 4.178 -9.9884 0.1457 18.3247 0.6395 1.5407 0.5524 0.337 0
SDindex Dindex SDbw
Number_clusters 2.0000 0 15.0000
Value_Index 2.6413 0 0.1491
$Best.partition
[1] 2 2 2 2 2 2 2 2 2 1 1 2 2 2 2 2 2 2 2 2 2 2 1 2 2 2 2 2 1 2 2 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 1 1 2
[54] 2 1 1 2 2 2 2 2 2 1 2 1 2 2 2 2 2 2 1 1 1 2 2 2 1 1 1 1 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 1 1 2

kmeans_output <- pca$x %>%
kmeans(centers = 3, nstart = 100)
combined_data <- cbind(task_map,
pca$x, factor(kmeans_output$cluster)) %>%
rename(cluster = `factor(kmeans_output$cluster)`)
fviz_eig(pca)

Standard Task Map Image with All Labels
p <- combined_data %>%
ggplot(aes(
x = PC1,
y = PC2,
label = task,
fill = cluster
)) + geom_point() + geom_label(nudge_y = 0.1, size = 4) +
#+ , alpha=0.05) +
# highlights only the ones in the selected set
# geom_label(
# data = subset(combined_data, task %in% c("NASA Moon survival", "Desert survival")),
# aes(
# x = PC1,
# y = PC2,
# label = task ,
# fill = cluster
# ),
# nudge_y = 0.1,
# size = 2
# )
theme_light(base_size = 24)
p # show the plot

ggsave(plot = p, filename = '../task-map.png')
Saving 28 x 10 in image
Task Map Image Highlighting Specific Subsets (for Illustrative
Purposes)
# An illustrative set to display
display_set <- c('Writing story',
'Advertisement writing',
'Desert survival',
'NASA Moon survival',
'Ultimatum game (various versions)',
'Dictator game and its variants',
'Prisoner\'s Dilemma (various versions)',
'9 Dot Problem',
'Word construction from a subset of letters',
'Typing game',
'Ravens Matrices',
'Euclidean traveling salesperson'
)
# A set of the tasks that are most different
max_diff_set <- c('Putting food into categories',
'9 Dot Problem',
'Shopping plan',
'Mock jury',
'Whac-A-Mole',
'Checkers',
'Reproducing arts',
'Image rating',
'TOPSIM - general mgmt business game',
'Word construction from a subset of letters',
'Minimal Group Paradigm (study diversity)')
# A set of tasks that are the most similar
min_diff_set <- c('Arithmetic problem 1',
'Euclidean traveling salesperson',
'Abstract grid task',
'Mastermind',
'Logic Problem',
'Guessing the correlation',
'Random dot motion',
'Letters-to-numbers problems (cryptography)',
'Computer maze',
'Recall images',
'Recall stories')
# A set of tasks that illustrates opportunities to add new tasks
display_limitations_set <- c('Recall word lists',
'Hidden figures in a picture (Recall Task)',
'Recall images',
'Recall stories',
'Recall videos',
'Writing story',
'Advertisement writing')
p <- combined_data %>%
ggplot(aes(
x = PC1,
y = PC2,
#label = task,
#fill = cluster
)) + geom_point(aes(size = 4)) +
#geom_point(aes(color = cluster, size = 4)) +
#highlights only the ones in the selected set
geom_point(data = subset(combined_data, task %in% display_limitations_set), aes(color = "firebrick1", size = 4)) +
geom_label(
data = subset(combined_data, task %in% display_limitations_set),
aes(
x = PC1,
y = PC2,
label = task
),
nudge_y = 0.1,
size = 4
) +
theme_minimal(base_size = 18) + theme(legend.position = "none")
p

ggsave(plot = p, filename = '../images/task-map_with_new_task_opportunities_highlighted.png')
Saving 28 x 10 in image
Create a cool 3D version
plot_ly(
x = combined_data$PC1,
y = combined_data$PC2,
z = combined_data$PC3,
type = "scatter3d",
mode = "markers", # can use mode = "text"
text = combined_data$task ,
color = combined_data$cluster
)
Create synthetic dependent variable based on the clusters
tasks_with_dv <- subset(combined_data, task %in% max_diff_set) %>%
mutate(
synergy = as.factor(ifelse(cluster == 3 | cluster == 2, 1, 0))
)
combined_data <- combined_data %>%
mutate(
synergy = as.factor(ifelse(cluster == 3 | cluster == 2, 1, 0))
)
Fitting and Visualizing Models for the Task Map.
x <- combined_data %>% select(PC1, PC2, synergy, task)
train <- tasks_with_dv %>% select(PC1, PC2, synergy, task)
model <- train %>% svm(synergy ~ PC1 + PC2, data = ., kernel = "linear")
svmplot <- decisionplot(model, x, class_var = "synergy") +
geom_point(data = train, aes(x = PC1, y = PC2, shape = synergy), color = "darkolivegreen2", show.legend = F) +
geom_label(data = train, aes(label = task ), nudge_y = 0.1, nudge_x = -0.1, size = 3) +
labs(title = "SVM (Linear Kernel)") +
theme_minimal(base_size = 12)
svmplot
ggsave('svmplot_synthetic_data.png')
Saving 7.29 x 4.51 in image

model <- train %>% knn3(synergy ~ PC1 + PC2, data = ., k = 1)
knnplot <- decisionplot(model, x, class_var = "synergy") +
geom_point(data = train, aes(x = PC1, y = PC2, shape = synergy), color = "darkolivegreen2", show.legend = F) +
geom_label(data = train, aes(label = task ), nudge_y = 0.1, nudge_x = -0.1, size = 3) +
labs(title = "kNN (1 neighbor)") +
theme_minimal(base_size = 12)
knnplot
ggsave('knnplot_synthetic_data.png')
Saving 7.29 x 4.51 in image

LS0tCnRpdGxlOiAiUiBOb3RlYm9vayBmb3IgUGFwZXItUmVsYXRlZCBWaXN1YWxpemF0aW9ucyIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKVGhpcyBub3RlYm9vayBjb250YWlucyB0aGUga2V5IHZpc3VhbGl6YXRpb25zIGZvciB0aGUgVGFzayBNYXBwaW5nIHBhcGVyLgoKYGBge3J9CmxpYnJhcnkoZmFjdG9leHRyYSkKbGlicmFyeShOYkNsdXN0KQpsaWJyYXJ5KGNsdXN0ZXIpCmxpYnJhcnkocGxvdGx5KQpsaWJyYXJ5KGdncGxvdDIpCmxpYnJhcnkoY2FyZXQpICNmb3Iga25uCmxpYnJhcnkoZTEwNzEpICNmb3Igc3ZtCmxpYnJhcnkoZHBseXIpCmxpYnJhcnkodGlkeXZlcnNlKQpgYGAKClZlcnkgdXNlZnVsIGRlY2lzaW9uIGJvdW5kYXJ5IHBsb3R0aW5nIGNvZGUgZnJvbTogaHR0cHM6Ly9taGFoc2xlci5naXRodWIuaW8vSW50cm9kdWN0aW9uX3RvX0RhdGFfTWluaW5nX1JfRXhhbXBsZXMvYm9vay9jbGFzc2lmaWNhdGlvbi1hbHRlcm5hdGl2ZS10ZWNobmlxdWVzLmh0bWwjay1uZWFyZXN0LW5laWdoYm9ycwpgYGB7ciBkZWNpc2lvbnBsb3R9CgpkZWNpc2lvbnBsb3QgPC0gZnVuY3Rpb24obW9kZWwsIGRhdGEsIGNsYXNzX3ZhciwgCiAgcHJlZGljdF90eXBlID0gYygiY2xhc3MiLCAicHJvYiIpLCByZXNvbHV0aW9uID0gNSAqIDc1KSB7CiAgIyByZXNvbHV0aW9uIGlzIHNldCB0byA3NSBkcGkgaWYgdGhlIGltYWdlIGlzIHJlbmRlcmVkICA1IGluY2VzIHdpZGUuIAogIAogIHkgPC0gZGF0YSAlPiUgcHVsbChjbGFzc192YXIpCiAgeCA8LSBkYXRhICU+JSBkcGx5cjo6c2VsZWN0KC1hbGxfb2YoY2xhc3NfdmFyKSkKICAKICAjIHJlc3Vic3RpdHV0aW9uIGFjY3VyYWN5CiAgcHJlZGljdGlvbiA8LSBwcmVkaWN0KG1vZGVsLCB4LCB0eXBlID0gcHJlZGljdF90eXBlWzFdKQogICMgTERBIHJldHVybnMgYSBsaXN0CiAgaWYoaXMubGlzdChwcmVkaWN0aW9uKSkgcHJlZGljdGlvbiA8LSBwcmVkaWN0aW9uJGNsYXNzCiAgcHJlZGljdGlvbiA8LSBmYWN0b3IocHJlZGljdGlvbiwgbGV2ZWxzID0gbGV2ZWxzKHkpKQogIAogIGNtIDwtIGNvbmZ1c2lvbk1hdHJpeChkYXRhID0gcHJlZGljdGlvbiwgcmVmZXJlbmNlID0geSkKICBhY2MgPC0gY20kb3ZlcmFsbFsiQWNjdXJhY3kiXQogIAogICMgZXZhbHVhdGUgbW9kZWwgb24gYSBncmlkCiAgciA8LSBzYXBwbHkoeFssIDE6Ml0sIHJhbmdlLCBuYS5ybSA9IFRSVUUpCiAgeHMgPC0gc2VxKHJbMSwxXSwgclsyLDFdLCBsZW5ndGgub3V0ID0gcmVzb2x1dGlvbikKICB5cyA8LSBzZXEoclsxLDJdLCByWzIsMl0sIGxlbmd0aC5vdXQgPSByZXNvbHV0aW9uKQogIGcgPC0gY2JpbmQocmVwKHhzLCBlYWNoID0gcmVzb2x1dGlvbiksIHJlcCh5cywgdGltZSA9IHJlc29sdXRpb24pKQogIGNvbG5hbWVzKGcpIDwtIGNvbG5hbWVzKHIpCiAgZyA8LSBhc190aWJibGUoZykKICAKICAjIyMgZ3Vlc3MgaG93IHRvIGdldCBjbGFzcyBsYWJlbHMgZnJvbSBwcmVkaWN0CiAgIyMjICh1bmZvcnR1bmF0ZWx5IG5vdCB2ZXJ5IGNvbnNpc3RlbnQgYmV0d2VlbiBtb2RlbHMpCiAgY2wgPC0gcHJlZGljdChtb2RlbCwgZywgdHlwZSA9IHByZWRpY3RfdHlwZVsxXSkKICAKICAjIExEQSByZXR1cm5zIGEgbGlzdAogIGlmKGlzLmxpc3QoY2wpKSB7IAogICAgcHJvYiA8LSBjbCRwb3N0ZXJpb3IKICAgIGNsIDwtIGNsJGNsYXNzCiAgfSBlbHNlCiAgICB0cnkocHJvYiA8LSBwcmVkaWN0KG1vZGVsLCBnLCB0eXBlID0gcHJlZGljdF90eXBlWzJdKSkKICAKICAjIHdlIHZpc3VhbGl6ZSB0aGUgZGlmZmVyZW5jZSBpbiBwcm9iYWJpbGl0eS9zY29yZSBiZXR3ZWVuIHRoZSAKICAjIHdpbm5pbmcgY2xhc3MgYW5kIHRoZSBzZWNvbmQgYmVzdCBjbGFzcy4KICAjIGRvbid0IHVzZSBwcm9iYWJpbGl0eSBpZiBwcmVkaWN0IGZvciB0aGUgY2xhc3NpZmllciBkb2VzIG5vdCBzdXBwb3J0IGl0LgogIG1heF9wcm9iIDwtIDEKICB0cnkoewogICAgbWF4X3Byb2IgPC0gdChhcHBseShwcm9iLCBNQVJHSU4gPSAxLCBzb3J0LCBkZWNyZWFzaW5nID0gVFJVRSkpCiAgICBtYXhfcHJvYiA8LSBtYXhfcHJvYlssMV0gLSBtYXhfcHJvYlssMl0KICB9LCBzaWxlbnQgPSBUUlVFKSAKICAKICBjbCA8LSBmYWN0b3IoY2wsIGxldmVscyA9IGxldmVscyh5KSkKICAKICBnIDwtIGcgJT4lIGFkZF9jb2x1bW4ocHJlZGljdGlvbiA9IGNsLCBwcm9iYWJpbGl0eSA9IG1heF9wcm9iKQogIAogIGdncGxvdChnLCBtYXBwaW5nID0gYWVzX3N0cmluZygKICAgIHggPSBjb2xuYW1lcyhnKVsxXSwKICAgIHkgPSBjb2xuYW1lcyhnKVsyXSkpICsKICAgIGdlb21fcmFzdGVyKG1hcHBpbmcgPSBhZXMoZmlsbCA9IHByZWRpY3Rpb24sIGFscGhhID0gcHJvYmFiaWxpdHkpKSArCiAgICAgZ2VvbV9jb250b3VyKG1hcHBpbmcgPSBhZXMoeiA9IGFzLm51bWVyaWMocHJlZGljdGlvbikpLCAKICAgICAgYmlucyA9IGxlbmd0aChsZXZlbHMoY2wpKSwgc2l6ZSA9IC41LCBjb2xvciA9ICJibGFjayIpICsKICAgIGdlb21fcG9pbnQoZGF0YSA9IGRhdGEsIG1hcHBpbmcgPSAgYWVzX3N0cmluZygKICAgICAgeCA9IGNvbG5hbWVzKGRhdGEpWzFdLAogICAgICB5ID0gY29sbmFtZXMoZGF0YSlbMl0sCiAgICAgIHNoYXBlID0gY2xhc3NfdmFyKSwgYWxwaGEgPSAuNykgKyAKICAgIHNjYWxlX2FscGhhX2NvbnRpbnVvdXMocmFuZ2UgPSBjKDAsMSksIGxpbWl0cyA9IGMoMCwxKSwgZ3VpZGUgPSAibm9uZSIpICsgIAogICAgbGFicyhzdWJ0aXRsZSA9IHBhc3RlKCJUcmFpbmluZyBhY2N1cmFjeToiLCByb3VuZChhY2MsIDIpKSkKfQpgYGAKCiMgTG9hZCB0aGUgRGF0YQpgYGB7cn0KdGFza19tYXAgPC0gcmVhZF9jc3YoJy4uL3Rhc2tfbWFwLmNzdicpCmBgYAoKIyBQbG90IHRoZSBUYXNrIE1hcCBhbmQgb3RoZXIgUmVsYXRlZCBJbWFnZXMKCkRyYXcgdGhlIHRhc2sgbWFwIHVzaW5nIFBDQSAmIGNsdXN0ZXJpbmcKCkZpcnN0LCBydW4gdGhlIFBDQQpgYGB7ciwgZmlnLndpZHRoPTE2LCBmaWcuaGVpZ2h0PTV9CnNldC5zZWVkKDEpCgpwY2EgPC0gdGFza19tYXAgJT4lICNzZWxlY3QoLWNvbnRpbnVvdXNfcXVlc3Rpb25zKSAlPiUKICBzZWxlY3QoLXRhc2spICU+JQogIHByY29tcChjZW50ZXIgPSBUKQoKIyBnZXQgb3B0aW1hbCBudW1iZXIgb2YgY2x1c3RlcnMgLS0gInNpbGhvdWV0dGUiIG1ldGhvZApmdml6X25iY2x1c3QoeCA9IHBjYSR4LCBGVU5jbHVzdGVyID0gc3RhdHM6OmttZWFucywgbWV0aG9kID0gInNpbGhvdWV0dGUiKSArCiAgbGFicyhzdWJ0aXRsZSA9ICJTaWxob3VldHRlIG1ldGhvZCIpCgojIGdldCBvcHRpbWFsIG51bWJlciBvZiBjbHVzdGVycwpOYkNsdXN0KGRhdGEgPSBwY2EkeCwgZGlzdGFuY2UgPSAiZXVjbGlkZWFuIiwKICAgICAgICBtaW4ubmMgPSAyLCBtYXgubmMgPSAxNSwgbWV0aG9kID0gImttZWFucyIpCgprbWVhbnNfb3V0cHV0IDwtIHBjYSR4ICU+JSAKICBrbWVhbnMoY2VudGVycyA9IDMsIG5zdGFydCA9IDEwMCkKCmNvbWJpbmVkX2RhdGEgPC0gY2JpbmQodGFza19tYXAsCiAgICAgIHBjYSR4LCBmYWN0b3Ioa21lYW5zX291dHB1dCRjbHVzdGVyKSkgJT4lCiAgcmVuYW1lKGNsdXN0ZXIgPSBgZmFjdG9yKGttZWFuc19vdXRwdXQkY2x1c3RlcilgKQoKZnZpel9laWcocGNhKQpgYGAKClN0YW5kYXJkIFRhc2sgTWFwIEltYWdlIHdpdGggQWxsIExhYmVscwpgYGB7ciwgZmlnLndpZHRoPTE0LCBmaWcuaGVpZ2h0PTV9CnAgPC0gY29tYmluZWRfZGF0YSAlPiUKICBnZ3Bsb3QoYWVzKAogICAgeCA9IFBDMSwKICAgIHkgPSBQQzIsCiAgICBsYWJlbCA9IHRhc2ssCiAgICBmaWxsID0gY2x1c3RlcgogICkpICsgZ2VvbV9wb2ludCgpICsgZ2VvbV9sYWJlbChudWRnZV95ID0gMC4xLCBzaXplID0gNCkgKwogIAogICMrICwgYWxwaGE9MC4wNSkgKwojIGhpZ2hsaWdodHMgb25seSB0aGUgb25lcyBpbiB0aGUgc2VsZWN0ZWQgc2V0CiAgIyBnZW9tX2xhYmVsKAogICMgICBkYXRhID0gc3Vic2V0KGNvbWJpbmVkX2RhdGEsIHRhc2sgJWluJSBjKCJOQVNBIE1vb24gc3Vydml2YWwiLCAiRGVzZXJ0IHN1cnZpdmFsIikpLAogICMgICBhZXMoCiAgIyAgICAgeCA9IFBDMSwKICAjICAgICB5ID0gUEMyLAogICMgICAgIGxhYmVsID0gdGFzayAsCiAgIyAgICAgZmlsbCA9IGNsdXN0ZXIKICAjICAgKSwKICAjICAgbnVkZ2VfeSA9IDAuMSwKICAjICAgc2l6ZSA9IDIKICAjICkKIHRoZW1lX2xpZ2h0KGJhc2Vfc2l6ZSA9IDI0KQoKcCAjIHNob3cgdGhlIHBsb3QKCmdnc2F2ZShwbG90ID0gcCwgZmlsZW5hbWUgPSAnLi4vdGFzay1tYXAucG5nJykKYGBgCgpUYXNrIE1hcCBJbWFnZSBIaWdobGlnaHRpbmcgU3BlY2lmaWMgU3Vic2V0cyAoZm9yIElsbHVzdHJhdGl2ZSBQdXJwb3NlcykKYGBge3IsIGZpZy53aWR0aD0xNCwgZmlnLmhlaWdodD01fQojIEFuIGlsbHVzdHJhdGl2ZSBzZXQgdG8gZGlzcGxheQpkaXNwbGF5X3NldCA8LSBjKCdXcml0aW5nIHN0b3J5JywKICdBZHZlcnRpc2VtZW50IHdyaXRpbmcnLCAKICdEZXNlcnQgc3Vydml2YWwnLAogJ05BU0EgTW9vbiBzdXJ2aXZhbCcsCiAnVWx0aW1hdHVtIGdhbWUgKHZhcmlvdXMgdmVyc2lvbnMpJywKICdEaWN0YXRvciBnYW1lIGFuZCBpdHMgdmFyaWFudHMnLAogJ1ByaXNvbmVyXCdzIERpbGVtbWEgKHZhcmlvdXMgdmVyc2lvbnMpJywKICc5IERvdCBQcm9ibGVtJywKICdXb3JkIGNvbnN0cnVjdGlvbiBmcm9tIGEgc3Vic2V0IG9mIGxldHRlcnMnLAogJ1R5cGluZyBnYW1lJywKICdSYXZlbnMgTWF0cmljZXMnLAogJ0V1Y2xpZGVhbiB0cmF2ZWxpbmcgc2FsZXNwZXJzb24nCiApCgojIEEgc2V0IG9mIHRoZSB0YXNrcyB0aGF0IGFyZSBtb3N0IGRpZmZlcmVudAptYXhfZGlmZl9zZXQgPC0gYygnUHV0dGluZyBmb29kIGludG8gY2F0ZWdvcmllcycsCiAnOSBEb3QgUHJvYmxlbScsCiAnU2hvcHBpbmcgcGxhbicsCiAnTW9jayBqdXJ5JywKICdXaGFjLUEtTW9sZScsCiAnQ2hlY2tlcnMnLAogJ1JlcHJvZHVjaW5nIGFydHMnLAogJ0ltYWdlIHJhdGluZycsCiAnVE9QU0lNIC0gZ2VuZXJhbCBtZ210IGJ1c2luZXNzIGdhbWUnLAogJ1dvcmQgY29uc3RydWN0aW9uIGZyb20gYSBzdWJzZXQgb2YgbGV0dGVycycsCiAnTWluaW1hbCBHcm91cCBQYXJhZGlnbSAoc3R1ZHkgZGl2ZXJzaXR5KScpCgojIEEgc2V0IG9mIHRhc2tzIHRoYXQgYXJlIHRoZSBtb3N0IHNpbWlsYXIKbWluX2RpZmZfc2V0IDwtIGMoJ0FyaXRobWV0aWMgcHJvYmxlbSAxJywKICdFdWNsaWRlYW4gdHJhdmVsaW5nIHNhbGVzcGVyc29uJywKICdBYnN0cmFjdCBncmlkIHRhc2snLAogJ01hc3Rlcm1pbmQnLAogJ0xvZ2ljIFByb2JsZW0nLAogJ0d1ZXNzaW5nIHRoZSBjb3JyZWxhdGlvbicsCiAnUmFuZG9tIGRvdCBtb3Rpb24nLAogJ0xldHRlcnMtdG8tbnVtYmVycyBwcm9ibGVtcyAoY3J5cHRvZ3JhcGh5KScsCiAnQ29tcHV0ZXIgbWF6ZScsCiAnUmVjYWxsIGltYWdlcycsCiAnUmVjYWxsIHN0b3JpZXMnKQoKIyBBIHNldCBvZiB0YXNrcyB0aGF0IGlsbHVzdHJhdGVzIG9wcG9ydHVuaXRpZXMgdG8gYWRkIG5ldyB0YXNrcwpkaXNwbGF5X2xpbWl0YXRpb25zX3NldCA8LSBjKCdSZWNhbGwgd29yZCBsaXN0cycsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ0hpZGRlbiBmaWd1cmVzIGluIGEgcGljdHVyZSAoUmVjYWxsIFRhc2spJywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnUmVjYWxsIGltYWdlcycsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ1JlY2FsbCBzdG9yaWVzJywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnUmVjYWxsIHZpZGVvcycsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ1dyaXRpbmcgc3RvcnknLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICdBZHZlcnRpc2VtZW50IHdyaXRpbmcnKQoKcCA8LSBjb21iaW5lZF9kYXRhICU+JQogIGdncGxvdChhZXMoCiAgICB4ID0gUEMxLAogICAgeSA9IFBDMiwKICAgICNsYWJlbCA9IHRhc2ssCiAgICAjZmlsbCA9IGNsdXN0ZXIKICAgICkpICsgZ2VvbV9wb2ludChhZXMoc2l6ZSA9IDQpKSArCiAgI2dlb21fcG9pbnQoYWVzKGNvbG9yID0gY2x1c3Rlciwgc2l6ZSA9IDQpKSArCiNoaWdobGlnaHRzIG9ubHkgdGhlIG9uZXMgaW4gdGhlIHNlbGVjdGVkIHNldApnZW9tX3BvaW50KGRhdGEgPSBzdWJzZXQoY29tYmluZWRfZGF0YSwgdGFzayAlaW4lIGRpc3BsYXlfbGltaXRhdGlvbnNfc2V0KSwgYWVzKGNvbG9yID0gImZpcmVicmljazEiLCBzaXplID0gNCkpICsKZ2VvbV9sYWJlbCgKICBkYXRhID0gc3Vic2V0KGNvbWJpbmVkX2RhdGEsIHRhc2sgJWluJSBkaXNwbGF5X2xpbWl0YXRpb25zX3NldCksCiAgYWVzKAogICAgeCA9IFBDMSwKICAgIHkgPSBQQzIsCiAgICBsYWJlbCA9IHRhc2sKICApLAogIG51ZGdlX3kgPSAwLjEsCiAgc2l6ZSA9IDQKKSArCiB0aGVtZV9taW5pbWFsKGJhc2Vfc2l6ZSA9IDE4KSArIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbiA9ICJub25lIikgIAoKcAoKZ2dzYXZlKHBsb3QgPSBwLCBmaWxlbmFtZSA9ICcuLi9pbWFnZXMvdGFzay1tYXBfd2l0aF9uZXdfdGFza19vcHBvcnR1bml0aWVzX2hpZ2hsaWdodGVkLnBuZycpCmBgYAoKQ3JlYXRlIGEgY29vbCAzRCB2ZXJzaW9uCmBgYHtyfQpwbG90X2x5KAogIHggPSBjb21iaW5lZF9kYXRhJFBDMSwKICB5ID0gY29tYmluZWRfZGF0YSRQQzIsCiAgeiA9IGNvbWJpbmVkX2RhdGEkUEMzLAogIHR5cGUgPSAic2NhdHRlcjNkIiwKICBtb2RlID0gIm1hcmtlcnMiLCAjIGNhbiB1c2UgbW9kZSA9ICJ0ZXh0IgogIHRleHQgPSBjb21iaW5lZF9kYXRhJHRhc2sgLAogIGNvbG9yID0gY29tYmluZWRfZGF0YSRjbHVzdGVyCikKYGBgCgpDcmVhdGUgc3ludGhldGljIGRlcGVuZGVudCB2YXJpYWJsZSBiYXNlZCBvbiB0aGUgY2x1c3RlcnMKYGBge3J9CnRhc2tzX3dpdGhfZHYgPC0gc3Vic2V0KGNvbWJpbmVkX2RhdGEsIHRhc2sgJWluJSBtYXhfZGlmZl9zZXQpICU+JQogIG11dGF0ZSgKICAgIHN5bmVyZ3kgPSBhcy5mYWN0b3IoaWZlbHNlKGNsdXN0ZXIgPT0gMyB8IGNsdXN0ZXIgPT0gMiwgMSwgMCkpCiAgKQpjb21iaW5lZF9kYXRhIDwtIGNvbWJpbmVkX2RhdGEgJT4lCiAgbXV0YXRlKAogICAgc3luZXJneSA9IGFzLmZhY3RvcihpZmVsc2UoY2x1c3RlciA9PSAzIHwgY2x1c3RlciA9PSAyLCAxLCAwKSkKICApCmBgYAoKIyBGaXR0aW5nIGFuZCBWaXN1YWxpemluZyBNb2RlbHMgZm9yIHRoZSBUYXNrIE1hcC4KCmBgYHtyfQp4IDwtIGNvbWJpbmVkX2RhdGEgJT4lIHNlbGVjdChQQzEsIFBDMiwgc3luZXJneSwgdGFzaykKdHJhaW4gPC0gdGFza3Nfd2l0aF9kdiAlPiUgc2VsZWN0KFBDMSwgUEMyLCBzeW5lcmd5LCB0YXNrKQptb2RlbCA8LSB0cmFpbiAlPiUgc3ZtKHN5bmVyZ3kgfiBQQzEgKyBQQzIsIGRhdGEgPSAuLCBrZXJuZWwgPSAibGluZWFyIikKCnN2bXBsb3QgPC0gZGVjaXNpb25wbG90KG1vZGVsLCB4LCBjbGFzc192YXIgPSAic3luZXJneSIpICsgCiAgZ2VvbV9wb2ludChkYXRhID0gdHJhaW4sIGFlcyh4ID0gUEMxLCB5ID0gUEMyLCBzaGFwZSA9IHN5bmVyZ3kpLCBjb2xvciA9ICJkYXJrb2xpdmVncmVlbjIiLCBzaG93LmxlZ2VuZCA9IEYpICsKICBnZW9tX2xhYmVsKGRhdGEgPSB0cmFpbiwgYWVzKGxhYmVsID0gdGFzayApLCBudWRnZV95ID0gMC4xLCBudWRnZV94ID0gLTAuMSwgc2l6ZSA9IDMpICsKICBsYWJzKHRpdGxlID0gIlNWTSAoTGluZWFyIEtlcm5lbCkiKSArCiAgdGhlbWVfbWluaW1hbChiYXNlX3NpemUgPSAxMikKCnN2bXBsb3QKICAKZ2dzYXZlKCdzdm1wbG90X3N5bnRoZXRpY19kYXRhLnBuZycpCmBgYAoKYGBge3J9Cm1vZGVsIDwtIHRyYWluICU+JSBrbm4zKHN5bmVyZ3kgfiBQQzEgKyBQQzIsIGRhdGEgPSAuLCBrID0gMSkKCmtubnBsb3QgPC0gZGVjaXNpb25wbG90KG1vZGVsLCB4LCBjbGFzc192YXIgPSAic3luZXJneSIpICsKICBnZW9tX3BvaW50KGRhdGEgPSB0cmFpbiwgYWVzKHggPSBQQzEsIHkgPSBQQzIsIHNoYXBlID0gc3luZXJneSksIGNvbG9yID0gImRhcmtvbGl2ZWdyZWVuMiIsIHNob3cubGVnZW5kID0gRikgKwogIGdlb21fbGFiZWwoZGF0YSA9IHRyYWluLCBhZXMobGFiZWwgPSB0YXNrICksIG51ZGdlX3kgPSAwLjEsIG51ZGdlX3ggPSAtMC4xLCBzaXplID0gMykgKwogIGxhYnModGl0bGUgPSAia05OICgxIG5laWdoYm9yKSIpICsgCiAgdGhlbWVfbWluaW1hbChiYXNlX3NpemUgPSAxMikKCmtubnBsb3QKICAKZ2dzYXZlKCdrbm5wbG90X3N5bnRoZXRpY19kYXRhLnBuZycpCmBgYA==